import requests
from lxml import etree
import pymysql.cursors
import configparser
from userAgent import getUserAgent
import time


# 获取每家医院的类
class getAllGetHospital:
    # 获得医院的信息
    def getHospital(self):
        # 配置头文件和需要爬取的url
        global hospital_name
        headers = getUserAgent()
        url = '''https://yyk.99.com.cn/henan'''

        # 爬取到指定的网页内容
        # allow_redirects=False可以防止重定向啥的问题
        get = requests.get(url, headers=headers, timeout=3000, allow_redirects=False)
        # 利用xpath解析html
        selector = etree.HTML(get.text)
        # 获取河南省每个市对应的医院信息
        items = selector.xpath('//div[@class="m-box"]')

        sum = 1  # 在控制输入插入了第几条的变量

        # 获取每个市的所有医院
        for e in items:
            # 获取具体的每个医院
            hospital = e.xpath('.//td')

            # 开始获取每个医院的信息
            for h_name in hospital:
                if len(e) != 0:
                    # href是医院具体的uri
                    hrefs = h_name.xpath('./a/@href')

                    url1 = '''https://yyk.99.com.cn'''  # 为了拼接字符串用的
                    headers = getUserAgent()  # 重新设定一下请求头，防止系统限制什么的
                    detailUrl = url1 + hrefs[0] + "jianjie.html"  # 获取每个医院的详细信息的url
                    contactUrl = url1 + hrefs[0] + "lianxi.html"  # 获取每个医院的联系方式的url

                    # 获取到了新的url，利用xpath进行重新解析(每个医院的详细信息)
                    detailHos = requests.get(detailUrl, headers=headers, timeout=5000, allow_redirects=False)
                    # 利用xpath解析医院详细信息的html
                    selector = etree.HTML(detailHos.text)

                    # 获取每个医院的封面
                    hospital_img = selector.xpath('//dl[@class="wrap-info"]/dt/a/img/@src')
                    if hospital_img:
                        hospital_img = str("https:" + hospital_img[0])
                    else:
                        hospital_img = ""

                    # 获取每个医院的详细信息(网页中那个表格中的所有内容)，用列表形式存储
                    hos_info = selector.xpath('//table[@class="present-table"]//td//text()')

                    # 排除表格中出现"-"的情况
                    for i in range(0, len(hos_info)):
                        if hos_info[i] == '-':
                            hos_info[i] = '0'

                    # 把表格中每个信息都对应一下
                    try:
                        hospital_name = str(hos_info[1])
                        hospital_area = str(hos_info[3])
                        hospital_dean_name = str(hos_info[5])
                        hospital_year = str(hos_info[7])
                        hospital_nature = str(hos_info[9])
                        hospital_grade = str(hos_info[11])
                        hospital_offices_num = int(hos_info[13])
                        medical_insurance_num = int(hos_info[15])
                        hospital_bed_num = int(hos_info[17])
                        outpatient_num = int(hos_info[19])
                        is_medical_insurance = str(hos_info[21])
                        print(hospital_name)
                    except:
                        print(hospital_name + "这个数据存在问题")

                    # 医院介绍
                    # /text() 是获取节点下的所有文字信息 //text()获取的包括标签的信息
                    hospital_about = selector.xpath('//div[@id="txtintro"]/text()')
                    # 去除字符串中的\n \r 空格等垃圾信息
                    if hospital_about:
                        hospital_about = str(hospital_about[0]).strip()

                    # 医院设备
                    hospital_equipment = selector.xpath('//div[@class="present-wrap2"]/p/text()')
                    # 去除字符串中的\n \r 空格等垃圾信息
                    if hospital_equipment:
                        hospital_equipment = str(hospital_equipment[0]).strip()
                    else:
                        hospital_equipment = ""

                    # 医院所获得的荣誉
                    hospital_honor = selector.xpath('//div[@class="present-wrap2 present-hx"]/p/text()')
                    # 去除字符串中的\n \r 空格等垃圾信息
                    if hospital_honor:
                        hospital_honor = str(hospital_honor[0]).strip()
                    else:
                        hospital_honor = ""

                    # 获取到了新的url，利用xpath进行重新解析(每个医院的详细信息)
                    contactHos = requests.get(contactUrl, headers=headers, timeout=500, allow_redirects=False)
                    # 利用xpath解析医院联系方式的html
                    selector = etree.HTML(contactHos.text)

                    # 获取医院的电话
                    hospital_phone = selector.xpath('/html/body/div[7]/div[3]/div[1]/div[1]/ul[1]/li[1]/font/text()')
                    if hospital_phone:
                        hospital_phone = str(hospital_phone[0])

                    else:
                        hospital_phone = ""

                    # 获取医院的邮编
                    hospital_post_code = selector.xpath(
                        '/html/body/div[7]/div[3]/div[1]/div[1]/ul[1]/li[4]/font/text()')
                    if hospital_post_code:
                        hospital_post_code = str(hospital_post_code[0])

                    else:
                        hospital_post_code = ""

                    # 获取医院的网址
                    hospital_url = selector.xpath('/html/body/div[7]/div[3]/div[1]/div[1]/ul[2]/li[1]/a/@href')
                    if hospital_url:
                        hospital_url = str(hospital_url[0])

                    else:
                        hospital_url = ""

                    # 获取医院的地址
                    hospital_address = selector.xpath('/html/body/div[7]/div[3]/div[1]/div[1]/ul[2]/li[2]/font/text()')
                    if hospital_address:
                        hospital_address = str(hospital_address[0])
                    else:
                        hospital_address = ""

                    # 获取去医院的公交路线
                    hospital_bus_route = selector.xpath('/html/body/div[7]/div[3]/div[1]/div[2]/p/text()')
                    if hospital_bus_route:
                        hospital_bus_route = str(hospital_bus_route[0])
                    else:
                        hospital_bus_route = ""

                    # 防止太频繁的爬网站
                    time.sleep(1)

                    try:
                        self.saveSql(hospital_name, hospital_area, hospital_img, hospital_dean_name, hospital_year,
                                     hospital_nature,
                                     hospital_grade, hospital_offices_num, medical_insurance_num, hospital_bed_num,
                                     outpatient_num,
                                     is_medical_insurance, hospital_equipment, hospital_about, hospital_honor,
                                     hospital_url,
                                     hospital_phone, hospital_address, hospital_post_code, hospital_bus_route)
                    except:
                        print("将数据写入sql时出错，但跳过异常，继续执行代码")

                    print("已插入%d条记录" % sum)
                    print("-----------------------------------")
                    sum += 1

    # 将爬取到的信息存入mysql数据库中
    @staticmethod
    def saveSql(hospital_name, hospital_area, hospital_img, hospital_dean_name, hospital_year, hospital_nature,
                hospital_grade, hospital_offices_num, medical_insurance_num, hospital_bed_num, outpatient_num,
                is_medical_insurance, hospital_equipment, hospital_about, hospital_honor, hospital_url, hospital_phone,
                hospital_address, hospital_post_code, hospital_bus_route):

        # configparser是用来读取配置文件的包
        cf = configparser.ConfigParser()
        # 得到配置文件
        cf.read("./config/config.ini")

        # 这里是
        config = {
            'host': cf.get("Mysql-Database", "host"),
            'port': int(cf.get("Mysql-Database", "port")),
            'user': cf.get("Mysql-Database", "user"),
            'password': cf.get("Mysql-Database", "password"),
            'db': cf.get("Mysql-Database", "db"),
            'charset': cf.get("Mysql-Database", "charset"),
            'cursorclass': pymysql.cursors.DictCursor,
        }

        # **config是以字典的方式接受参数
        connection = pymysql.connect(**config)
        # 获取操作的游标
        curs = connection.cursor()
        # 定义的sql语句
        sql = 'INSERT INTO hospital(hospital_name,hospital_area,hospital_img,hospital_dean_name,hospital_year,hospital_nature,hospital_grade,hospital_offices_num,medical_insurance_num,hospital_bed_num,outpatient_num,is_medical_insurance,hospital_equipment,hospital_about,hospital_honor,hospital_url,hospital_phone,hospital_address,hospital_post_code,hospital_bus_route) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'

        curs.execute(sql, (
            hospital_name, hospital_area, hospital_img, hospital_dean_name, hospital_year, hospital_nature,
            hospital_grade, hospital_offices_num, medical_insurance_num, hospital_bed_num, outpatient_num,
            is_medical_insurance, hospital_equipment, hospital_about, hospital_honor, hospital_url, hospital_phone,
            hospital_address, hospital_post_code, hospital_bus_route))

        connection.commit()

        curs.close()
        connection.close()


if __name__ == '__main__':
    # 初始化类
    start = getAllGetHospital()
    start.getHospital()
